#!/bin/bash
#SBATCH -J GROMACS-DCU
#SBATCH -p wzhdnormal
#SBATCH -N 1
#SBATCH --ntasks-per-node=4
#SBATCH --cpus-per-task=8
#SBATCH --gres=dcu:4
#SBATCH -o log/many_submit_%j.loop
#SBATCH -e log/many_submit_%j.loop

source ~/.bashrc
#source /work/home/aixplorerbio_wz/miniconda3/bin/activate amber
#source /work/home/aixplorerbio_wz/software/gromacs-2020.3/bin/GMXRC

module switch compiler/dtk/21.04
module load apps/Gromacs-DCU2/2020.3-dtk21.04/hpcx-gcc-7.3.1

# 此脚本用于单独跑不同pdb的单个md任务, 最多支持4个md任务
# 如果小于4个, 可以同时修改 --ntasks-per-node=N, --gres=dcu:N (1<=N<=4) 已达到资源利用率最高

pwd=$(pwd)

if [ $# == 0 ]; then
    echo "USAGE: $0 MD_PATH1 <MD_PATH2 ..>"
    echo " e.g.: $0 test_md/"
    exit 1
fi

i=0

md_num=$#

if (($md_num > 4)); then
    md_num = 4
fi

per_cpu=7

for p in $*; do
    echo "$i params = $p"

    if [ -d "$pwd/$p" ]; then
        cd $pwd/$p

        export GOMP_CPU_AFFINITY="$(($i * $per_cpu))-$((($i + 1) * $per_cpu))"
        echo "start gmx work in $p use gpu_id $i .. use cpu_num $per_cpu,  $GOMP_CPU_AFFINITY"

        mpirun -np 1 gmx_mpi mdrun -gpu_id $i -ntomp $(($per_cpu + 1)) -nb gpu -pme gpu -v --deffnm md -cpi md.cpt -g test.log &
    fi

    i=$(($i + 1))
    if (($i == 4)); then
        break
    fi

done

echo "start wait ..."
wait

echo "done"

# mpirun -hostfile hostfile -np 4 -N 4 --bind-to none $PROJECT_HOME/work_dir/one_bind.sh
